/**
 * 
 */
package com.analysis;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;

import org.joy.analyzer.html.ParseException;

import com.crawl.MetaSearch;
import com.search.Similarity;

/**
 * @author LvSaiHui {gurnfe@163.com}
 * @version 1.0 <br>
 *          Copyright (C), 2007-2008, ZJUT <br>
 *          This program is protected by copyright laws. <br>
 *          Program Name: BaiDuAnalysis.java <br>
 *          Date: 2009-3-22 <br>
 *          Description:
 */
public class Analysis {

	/**
	 * @param args
	 * @throws IOException
	 * @throws ParseException
	 */
	/**
	 * @param args
	 * @throws IOException
	 * @throws ParseException
	 */
	public static void main(String[] args) throws IOException, ParseException {
		String filename = "D:\\eclipse_workplace\\crawler\\src\\urls.txt";
		InputStreamReader read = new InputStreamReader(new FileInputStream(
				filename), "UTF-8");
		BufferedReader reader = new BufferedReader(read);
		String buffer;
		List<String> crawledUrls = new ArrayList<String>();
		while ((buffer = reader.readLine()) != null
				&& buffer.trim().length() > 0) {
			crawledUrls.add(buffer);
		}
		if(reader!=null){
			reader.close();
		}
		ArrayList<String> baiduUrls = (ArrayList<String>) MetaSearch
				.getBaiduUrls("信用", 1000);
		int baiduRelated = 0;
		int googleRelated = 0;
		int sagaRelated = 0;
		//统计主题相关度比例和已爬取网页数量的关系
		for (int i = 0; i < 1000 && i < baiduUrls.size()
				&& i < crawledUrls.size(); i++) {
			if (Similarity.scoreOnWebPageText(baiduUrls.get(i)) > 0.28) {
				baiduRelated++;
			}
			if (Similarity.scoreOnWebPageText(crawledUrls.get(i)) > 0.28) {
				sagaRelated++;
			}
			if (i % 100 == 0) {
				if (i == 100) {
					System.out.println("100 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 100.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 100.0);

				}
				if (i == 200) {
					System.out.println("200 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 200.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 200.0);

				}
				if (i == 300) {
					System.out.println("300 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 300.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 300.0);

				}
				if (i == 400) {
					System.out.println("400 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 400.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 400.0);

				}
				if (i == 500) {
					System.out.println("500 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 500.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 500.0);

				}
				if (i == 600) {
					System.out.println("600 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 600.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 600.0);

				}
				if (i == 700) {
					System.out.println("700 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 700.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 700.0);

				}
				if (i == 800) {
					System.out.println("800 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 800.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 800.0);

				}
				if (i == 900) {
					System.out.println("900 webpage baiduRelated : " + baiduRelated + "-"
							+ baiduRelated / 900.0 + ",sagaRelated :"
							+ sagaRelated + "-" + sagaRelated / 900.0);

				}
				

			}
		}
		 baiduRelated = 0;
		 googleRelated = 0;
		 sagaRelated = 0;
		 //统计平均主题相关度比例和已爬取网页数量的关
		 for(int i=0;i<100;i++){
			 if (Similarity.scoreOnWebPageText(baiduUrls.get(i)) > 0.28) {
					baiduRelated++;
				}
				if (Similarity.scoreOnWebPageText(crawledUrls.get(i)) > 0.28) {
					sagaRelated++;
				} 
		 }

	}

}
